package com.strat.searcher.analisys.tokenize;

import java.io.File;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.ibm.icu.text.BreakIterator;
import com.ibm.icu.text.RuleBasedBreakIterator;
import com.strat.searcher.exceptions.TextAnalisysException;

/**
 * Class tokenize a block of text into its component words. Decorator of
 * com.ibm.icu.text.RuleBasedBreakIterator.
 * 
 * @author Oleksandr Stratiyevskyy
 * @version $Revision$
 */
public class TextTokenizer {

    private final Logger logging = LoggerFactory.getLogger(this.getClass());
    private String text;
    private int index = 0;
    private RuleBasedBreakIterator breakIterator;

    @SuppressWarnings("unchecked")
    private final static Map<Integer, TokenType> RULE_ENTITY_MAP = ArrayUtils
            .toMap(new Object[][] { { new Integer(0), TokenType.UNKNOWN },
                    { new Integer(100), TokenType.NUMBER },
                    { new Integer(200), TokenType.WORD },
                    { new Integer(500), TokenType.ABBREVIATION },
                    { new Integer(501), TokenType.WORD },
                    { new Integer(504), TokenType.MARKUP }, });

    public TextTokenizer() {
        this("src/main/resources/word_break_rules.txt");
    }

    public TextTokenizer(String rulesFilename) {
        super();
        initBreakIterator(rulesFilename);
    }

    private void initBreakIterator(String rulesFilename) {
        try {
            this.breakIterator = new RuleBasedBreakIterator(
                    FileUtils
                            .readFileToString(new File(rulesFilename), "UTF-8"));
        } catch (IOException e) {
            logging.error("", e);
            throw new TextAnalisysException();
        }
    }

    public void setText(String text) {
        this.text = text;
        this.breakIterator.setText(text);
        this.index = 0;
    }

    public String getText() {
        return text;
    }

    public Token nextToken() {
        while (true) {
            int end = breakIterator.next();
            if (end == BreakIterator.DONE) {
                return null;
            }
            String nextWord = text.substring(index, end);
            index = end;
            return new Token(nextWord, RULE_ENTITY_MAP.get(breakIterator
                    .getRuleStatus()));
        }
    }
}
